#!/usr/python/bin
#coding:utf-8
#Filename:NltkTest59,一些关于语料库使用的测试
import nltk
from nltk.corpus import brown
from nltk.corpus import reuters
from nltk.corpus import inaugural
from nltk.corpus import udhr
from nltk.corpus import BracketParseCorpusReader
from nltk.corpus import PlaintextCorpusReader
 
class NltkTest59:

    def __init__(self):
        print 'Initing...'
    def BrownTest(self,genres,modals):
        '''来源于p59,对于不同问题的常用词统计的测试'''
        cfd = nltk.ConditionalFreqDist(\
            (genre, word)\
            for genre in brown.categories()\
            for word in brown.words(categories=genre))
        cfd.tabulate(conditions=genres,samples=modals)
    def ReutersTest(self):
        # reuters.fileids()
        # reuters.categories()
        print reuters.categories('training/9865')
        print reuters.categories(['training/9865', 'training/9880'])
        # reuters.fileids('barley')
        # reuters.fileids(['barley', 'corn'])
        print reuters.words('training/9865')[:14]
        print reuters.words(['training/9865','training/9880'])
        print reuters.words(categories='barley')
        print reuters.words(categories=['barley', 'corn'])
    def InauguralTest(self):
        '''运行会出错 '''
        cfd=nltk.ConditionalFreqDist(\
            (target, file[:4])\
            for fileids in inaugural.fileids()\
            for w in inaugural.words(fileids)\
            for target in ['america','citizen']\
            if w.lower().startswith(target))
        cfd.plot()
    def UdhrTest(self):
        languages = ['Chickasaw', 'English','German_Deutsch',\
                     'Greenlandic_Inuktikut','Hungarian_Magyar', 'Ibibio_Efik']
        cfd=nltk.ConditionalFreqDist(\
            (lang, len(word))\
            for lang in languages\
            for word in udhr.words(lang +'-Latin1'))
        cfd.plot(cumulative=False)
    def CropsPCRTest(self):
        corpus_root=r'C:\corpora\udhr2'
        file_pattern=r'.*'
        encoding='utf-8'
        pcr=PlaintextCorpusReader(corpus_root,file_pattern)
        print pcr.fileids()
        print pcr.words('007.txt')
    def CropsBPCRTest(self):
        '''可耻的失败了，没法运行，可能是语料库版本问题，或者其他，待查'''
       #corpus_root=r'C:\corpora\penntreebank\parsed'
        corpus_root=r'/home/mqx/osc/pycode/searchEngine/cnki/'
        file_pattern=r'*.wsj'
        ptb=BracketParseCorpusReader(corpus_root, file_pattern)
        print ptb.fileids()
        print len(ptb.sents())
        ptb.sents(fileids='\wsj_0001.mrg')[19]
 
       
      
nt59=NltkTest59()
genres= ['news', 'religion', 'hobbies', 'science_fiction', 'romance', 'humor']
modals= ['can', 'could', 'may', 'might', 'must', 'will']
#nt59.BrownTest(genres,modals)
nt59.ReutersTest()
#不能运行，因为存在SOAP版本错误，不急着解决，以后再补一个解决方案
#nt59.InauguralTest()
#nt59.UdhrTest()
#nt59.CropsPCRTest()
#nt59.CropsBPCRTest()
